package services;

import helpers.NumberHelper;
import helpers.StringHelper;
import helpers.UrlHelper;

import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.Future;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.mail.internet.NewsAddress;

import models.IpAddressModel;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.Connection.Response;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPMethod;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
import com.google.gson.Gson;

import debugs.DomainDebug;
import debugs.IpAddressDebug;

import entities.Domain;
import entities.IpAddress;

public class GetInfoDomain {
	
	private Domain domain = new Domain();
	
	private TreeMap<String, String> check = new TreeMap<String, String>();
	
	private URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
	
	private Future fAlexa = null;
	
	private Future fWhois = null;
	
	private Future fIpAddress = null;
	
	private Future fWhoisIp = null;
	
	private Future fInfoIp = null;
	
	private Future fGoogleRank = null;
	
	private Future fAgeDomain = null;
	
	private Future fValidator = null;
	
	private Future fDns = null;
	
	private Future fGoogleService = null;
	
	private List<String> googleAnalyticSites = new ArrayList<String>();
    
    private List<String> googleAdsenceSites = new ArrayList<String>();
    
    private List<String> siteSameIp = new ArrayList<String>();
	
	public Domain getDomain() {
		return domain;
	}

	public TreeMap<String, String> getCheck() {
        return check;
    }
	
	/********************************************* MAIN PROCESS *********************************************/
	public GetInfoDomain(Domain domain) {
		this.domain = domain;

	}
	
	public void getStep1() {
		//Load content from urls
		loadUrlAlexa();
		
		loadUrlWhois();
		
		loadUrlIpAddress();
		
		loadUrlGoogleRank();
		
		loadUrlAgeDomain();
		
		loadUrlValidator();
		
		loadUrlDns();
		
		//Get content form url loaded
		fAlexaService();
		
		getWhois();
		
		getIpAddress();
		
		getGooglePageRank();
		
		getAgeDomain();
		
		getValidator();
		
		getDns();
	}
	
	/********************************************* SITE INFO *********************************************/
	
	public void getSiteInfo() {
		String url = "http://" + domain.getDomain();
		try {
			Connection conn = Jsoup.connect(url);
			Document doc = conn
					.header("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
					.header("accept-Language", "en-us,en;q=0.5")
					.userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:15.0) Gecko/20100101 Firefox/15.0.1")
					.followRedirects(true)
					.timeout(3000)
					.get();
			Response response = conn.response();
			
            //Get site header
			String headers = new Gson().toJson(response.headers());
			domain.setHeaders(headers);
			
			//Set server
			if (response.headers().get("server") != null) {
				domain.setWebServer(response.headers().get("server"));
			}
			
			//Get title, keywords, description
			String title = "";
            String description = "";
            String keywords  = "";
            
            try {
                title = doc.select("title").first().text();
            } catch (Exception e) {
                this.check.put("title", "Can't get info title");
            }

            try {
                keywords = doc.select("meta[name=keywords]").first().attr("content");
            } catch (Exception e) {
                try {
                    keywords = doc.select("meta[http-equiv=keywords]").first().attr("content");
                } catch (Exception e2) {
                    this.check.put("keywords", "Can't get info keywords");
                }        
            }
            
            try {
                description = doc.select("meta[name=description]").first().attr("content");
            } catch (Exception e) {
                try {
                    description = doc.select("meta[http-equiv=description]").first().attr("content");
                } catch (Exception e2) {
                    if(doc.select("body").first().text().length() > 260)
                    {
                        description = doc.select("body").first().text().substring(0,250).replaceAll("[ ]+[\\w\\W]{0,10}+$", "")+" ...";
                    } else {
                        description = doc.select("body").first().text();
                    }
                }
            }
			
            //Set title, keywords, description
            this.domain.setTitle(title);
            this.domain.setKeywords(keywords);
            this.domain.setDescription(description);
            
            String content = conn.get().html();
            
            //Set size
            long size = Long.parseLong(Integer.toString(content.getBytes().length));
            this.domain.setPageSize(NumberHelper.readableFileSize(size));
            
            //Get Google Analytic
            getGoogleAnalytic(content);
            
            //Get Google Adsense
	        getGoogleAdsense(content);
	        
	        //Get javascript library
	        getJavascriptLibrary(content);
	        
	        //Get heading
	        getHeading(doc);
	        
	        //Get images
	        getImage(doc);
			
	        //Get text on html
	        getTextOnHtml(content);
	        
	        //Get out link
	        getOutlink(content);
	        
		} catch (Exception e) {
			
			domain.setExists(false);
		}
	}
	
	private void getGoogleAnalytic(String content) {
        int begin = content.indexOf("_gaq.push(['_setAccount', '");
        String analyticAccount = "";
        String googleAnalyticPropertyId = "";
        if(begin>0)
        {
            begin+="_gaq.push(['_setAccount', '".length();
            int end =  content.indexOf("'",begin);
            analyticAccount = content.substring(begin, end);
            googleAnalyticPropertyId = content.substring(begin, end);
            begin=analyticAccount.lastIndexOf("-")-1;
            analyticAccount = analyticAccount.substring(0, begin+1);
        }
        else
        {
             begin = content.indexOf("_gaq.push([\"_setAccount\", \"");
            if(begin>0)
            {
                begin+="_gaq.push([\"_setAccount\", \"".length();
                int end =  content.indexOf("\"",begin);
                analyticAccount = content.substring(begin, end);
                googleAnalyticPropertyId = content.substring(begin, end);
            } else {
                begin = content.indexOf("_uacct");
                if(begin > 0)
                {
                    begin+="_uacct".length();
                    analyticAccount = content.substring(begin,begin+30).replaceAll("^[ =\\'\\\"]+", "").replaceAll("[^a-zA-Z0-9\\-\\_]+[\\w\\W]+", "");
                    googleAnalyticPropertyId = analyticAccount;
                    begin=analyticAccount.lastIndexOf("-");
                    
                    analyticAccount = analyticAccount.substring(0, begin);
                } else {
                    Pattern p = Pattern.compile("UA-[0-9]+");

                    Matcher m = p.matcher(content);
                    while (m.find()) {
                        analyticAccount = m.group();
                        googleAnalyticPropertyId = analyticAccount;
                    }
                }
            }
        }
        
        this.domain.setGoogleAnalyticId(analyticAccount);
        this.domain.setGoogleAnalyticPropertyId(googleAnalyticPropertyId);
    }
	
	private void getGoogleAdsense(String content) {
        int begin = content.indexOf("google_ad_client = \"");
        String adsAccount = "";
        if( begin > 0 )
        {
            begin+="google_ad_client = \"', '".length();
            int end =  content.indexOf("\"",begin);
            adsAccount = content.substring(begin, end);
        }
        
        this.domain.setGoogleAdsenseId(adsAccount);
    }
	
	private void getJavascriptLibrary(String content) {
        List<String> jsLib = new ArrayList<String>();
        if(content.indexOf("jquery")>0)
            jsLib.add("jquery");
        
        if(content.indexOf("prototype.js")>0)
            jsLib.add("prototype");
        
        if(content.indexOf("extjs")>0)
            jsLib.add("extjs");
        
        if(content.indexOf("yui")>0)
            jsLib.add("yui");
        
        if(content.indexOf("dojo")>0)
            jsLib.add("dojo");
        
        if(content.indexOf("gwt")>0)
            jsLib.add("gwt");

        if(content.indexOf("easeljs")>0)
            jsLib.add("easeljs");
        
        this.domain.setJavascriptLibrary(new Gson().toJson(jsLib));
    }
	
	private void getHeading(Document doc) {
    	try {
            Elements h1 = doc.select("H1");
            Elements h2 = doc.select("H2");
            Elements h3 = doc.select("H3");
            Elements h4 = doc.select("H4");
            Elements h5 = doc.select("H5");
            Elements h6 = doc.select("H6");
            
            Map<String, String> heading = new TreeMap<String, String>();
            heading.put("H1", Integer.toString(h1.size()));
            heading.put("H2", Integer.toString(h2.size()));
            heading.put("H3", Integer.toString(h3.size()));
            heading.put("H4", Integer.toString(h4.size()));
            heading.put("H5", Integer.toString(h5.size()));
            heading.put("H6", Integer.toString(h6.size()));
            
            this.domain.setHeading(new Gson().toJson(heading));
            
        } catch (Exception e) {
            check.put("heading", "Can't get heading of domain " + domain.getDomain());
        }
    }
	
	private void getImage(Document doc) {
		try {
			Elements png = doc.select("img[src$=.png]");
			Elements jpg = doc.select("img[src$=.jpg]");
			Elements gif = doc.select("img[src$=.gif]");
			
			Map<String, String> images = new TreeMap<String, String>();
			images.put("PNG", Integer.toString(png.size()));
			images.put("JPG", Integer.toString(jpg.size()));
			images.put("GIF", Integer.toString(gif.size()));
			
			this.domain.setImages(new Gson().toJson(images));
		} catch (Exception e) {
			check.put("images", "Can't get images of domain " + domain.getDomain());
		}
	}
	
	private void getTextOnHtml(String content) {
    	try {
			int textLength = content.length();
			int htmlLength = content.replaceAll("\\<.*?>","").length();
			double d = (double) htmlLength/textLength;
			
			this.domain.setRatioTextHtml(1-d);
		} catch (Exception e) {
			check.put("ratioTextHtml", "Can't get ratioTextHtml of domain " + domain.getDomain());
		}
    }
	
	/********************************************* LOAD URL *********************************************/
	private void loadUrlAlexa() {
		String urlAlexa = "http://www.alexa.com/siteinfo/";
        try {
            URL url = new URL(urlAlexa + domain.getDomain());
            fAlexa = fetcher.fetchAsync(url);
        } catch (Exception e) {
            check.put("fAlexa", "Can't run fAlexa");
        }
    }
	
	private void loadUrlWhois() {
        try {
            URL urlWhois = new URL("http://www.whoisdata.us/whois/" + this.domain.getDomain());
            fWhois = fetcher.fetchAsync(urlWhois);
        } catch (Exception e) {
            check.put("fWhois", "Can't run fWhois");
        }
    }
	
	private void loadUrlIpAddress() {
        try {
            URL urlIpAddress = new URL("http://www.nearby.org.uk/api/dig.php?l=" + this.domain.getDomain());
            fIpAddress = fetcher.fetchAsync(urlIpAddress);
        } catch (Exception e) {
            check.put("fIpAddress", "Can't run fIpAddress");
        }
    }
	
	private void loadUrlWhoisIp() {
    	try {
            URL urlWhoisIp = new URL("http://www.who.is/whois-ip/ip-address/" + domain.getIpAddress() + "/");
            fWhoisIp = fetcher.fetchAsync(urlWhoisIp);
            
        } catch (Exception e) {
            check.put("fWhoisIp", "Can't run fWhoisIp");
        }
    }
	
	private void loadUrlInfoIp() {
    	try {
            URL urlInfoIp = new URL("http://whatismyipaddress.com/ip/" + domain.getIpAddress());
            fInfoIp = fetcher.fetchAsync(urlInfoIp);
            
        } catch (Exception e) {
            check.put("fInfoIp", "Can't run fInfoIp");
        }
    }
	
	private void loadUrlGoogleRank() {
        try {
            URL urlGoogleRank = new URL("http://www.gpagerank.net/?url=" + domain.getDomain());
            fGoogleRank = fetcher.fetchAsync(urlGoogleRank);
        } catch (Exception e) {
            check.put("fGoogleRank", "Can't run fGoogleRank");
        }
    }
	
	private void loadUrlAgeDomain() {
        try {
            URL urlAgeDomain = new URL("http://www.seologs.com/dns/domain-check.html");
            HTTPRequest request = new HTTPRequest(urlAgeDomain, HTTPMethod.POST);
            request.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
            String params = "query=" + domain.getDomain() + "&output=cdt";
            request.setPayload(params.getBytes());
            fAgeDomain = fetcher.fetchAsync(request);

        } catch (Exception e) {
            check.put("fAgeDomain", "Can't run fAgeDomain");
        }
    }
	
	private void loadUrlValidator() {
        try {
            URL urlValidator = new URL("http://validator.w3.org/check?uri=http://" + domain.getDomain());
            fValidator = fetcher.fetchAsync(urlValidator);
            
        } catch (Exception e) {
            check.put("fValidator", "Can't run fValidator");
        }
    }
	
	private void loadUrlDns() {
    	try {
            URL urlDns = new URL("http://dns.violet.edu.vn/dns/" + domain.getDomain());
            fDns = fetcher.fetchAsync(urlDns);
            
        } catch (Exception e) {
            check.put("fDns", "Can't run fDns");
        }
    }
	
	/********************************************* GET CONTENT *********************************************/
	private void fAlexaService() {
        try {
            HTTPResponse response = (HTTPResponse) fAlexa.get();
            
            String content = this.getContentCharset(response);
            
            getAlexaRank(content);
            getDailyVisitor(content);
            getCountryVisitor(content);
            
        } catch (Exception e) {
            check.put("fAlexa", "Can't response fAlexa");
        }
    }
	
	private void getAlexaRank(String content) {
        try {
            Document doc = Jsoup.parse(content);
            String strAlexaRank = doc.select("#siteStats").select("tbody tr td div").first().text().trim().replace(",","");
            int alexaRank = 0;
            
            try {
                if(strAlexaRank != null && !strAlexaRank.isEmpty())
                {
                    alexaRank=Integer.parseInt( strAlexaRank );
                }
            } catch (Exception e) {
                check.put("alexaGlobeRank", "Can't get position alexa rank of domain " + domain.getDomain());
            }
            
            this.domain.setAlexaRank(alexaRank);
            
        } catch (Exception e) {
            check.put("alexaRank", "Can't get position alexa rank of domain " + domain.getDomain());
        }
    }
	
	private void getDailyVisitor(String content) {
        try {
            Document doc = Jsoup.parse(content);
            String strVisitor = doc.select("#reach .visitors_percent").select("tr td.avg").first().text().replaceAll("[^0-9.]*", "");
            int dailyVisitor = 0;
            
            try {
                if(strVisitor != null && !strVisitor.isEmpty())
                {
                    dailyVisitor = (int) Math.round(((350 * Float.parseFloat(strVisitor)) * 100) / 0.03);
                }
            } catch (Exception e) {
                check.put("dailyVisitor", "Can't get daily visitor of domain " + domain.getDomain());
            }
            
            this.domain.setDailyVisitor(dailyVisitor);
            
        } catch (Exception e) {
            check.put("dailyVisitor", "Can't get daily visitor of domain " + domain.getDomain());
        }
    }
	
	private void getCountryVisitor(String content) {
        try {
            Document doc = Jsoup.parse(content);
            Elements countries = doc.select("#visitors-by-country .tr1");
            
            Map<String, String> visitor = new TreeMap<String, String>();
            
            
            for (Element country : countries) {
                try {
                    String name = country.select(".tc1").get(0).select("a").html().replaceAll("<[^>]+/>", "").replace("&nbsp;", "").trim();
                    String percent = country.select(".tc1").get(1).html().replace("%", "");
                    visitor.put(name, percent);
                } catch (Exception e) {
                }
            }
            String visitors = new Gson().toJson(visitor);
            this.domain.setCountryVisitor(visitors);
        } catch (Exception e) {
            check.put("countryVisitor", "Can't get visitor by country of domain " + domain.getDomain());
        }
    }
	
	private void getWhois() {
        try {
            HTTPResponse response = (HTTPResponse) fWhois.get();
            Document doc = Jsoup.parse(this.getContentCharset(response));
            String whois = doc.select("#content .whois").html();
            if (!whois.isEmpty()) {
            	whois = whois.replaceAll("[<]{1}(a)[^>]+[>]{1}", "").replaceAll("<\\/a>", "");
				//whois = whois.replaceAll("(src=\")", "src=\"http://www.whoismind.com");
            	this.domain.setWhois(whois);
            }
        } catch (Exception e) {
            check.put("fWhois", "Error when get value whois");
        }
    }
	
	private void getIpAddress() {
        try {
            HTTPResponse response = (HTTPResponse) fIpAddress.get();
            this.domain.setIpAddress(this.getContentCharset(response));
            
            loadUrlWhoisIp();
            //loadUrlInfoIp();
            
            getWhoisIp();
            //getInfoIp();
        } catch (Exception e) {
            check.put("ipAddress", "Can't get ipAddress for this domain " + domain.getDomain());
        }
    }
	
	private void getWhoisIp() {
    	try {
			HTTPResponse response = (HTTPResponse) fWhoisIp.get();
			Document doc = Jsoup.parse(this.getContentCharset(response));
			if (doc.select("#domain-data .tab-pane .domain-data pre") != null) {
				String whoisIp = doc.select("#domain-data .tab-pane .domain-data pre").html();
				this.domain.setWhoisIp(whoisIp);
			}
			
		} catch (Exception e) {
			check.put("whoisIp", "Can't get whois ip info for domain " + domain.getIpAddress());
		}
    }
	
	private void getInfoIp() {
		try {
			HTTPResponse response = (HTTPResponse) fInfoIp.get();
			Document doc = Jsoup.parse(this.getContentCharset(response));
			
			if (domain.getIpAddress() != null && !domain.getIpAddress().isEmpty()) {
				IpAddressModel ipAddressModel = IpAddressModel.getInstance();
				IpAddress ipAddress = ipAddressModel.get(domain.getIpAddress());
				
				if (ipAddress == null) {
					ipAddress = new IpAddress();
					ipAddress.setIp(domain.getIpAddress());
				}
				
				String decimal = doc.select("#section_content table:eq(0) tr:eq(1) td").html();
				String hostname = doc.select("#section_content table:eq(0) tr:eq(2) td").html();
				String isp = doc.select("#section_content table:eq(0) tr:eq(3) td").html();
				String organization = doc.select("#section_content table:eq(0) tr:eq(4) td").html();
				String services = doc.select("#section_content table:eq(0) tr:eq(5) td").html();
				String type = doc.select("#section_content table:eq(0) tr:eq(6) td").html();
				type = type.replaceAll("[<]{1}(a)[^>]+[>]{1}", "").replaceAll("<\\/a>", "");
				
				String assignment = doc.select("#section_content table:eq(0) tr:eq(7) td").html();
				assignment = assignment.replaceAll("[<]{1}(a)[^>]+[>]{1}", "").replaceAll("<\\/a>", "");
				
				String country = doc.select("#section_content > table tr:eq(0) td").html();
				String region = doc.select("#section_content > table tr:eq(1) td").html();
				String city = doc.select("#section_content > table tr:eq(2) td").html();
				String latitude = doc.select("#section_content > table tr:eq(3) td").html();
				String longitude = doc.select("#section_content > table tr:eq(4) td").html();
				String areaCode = doc.select("#section_content > table tr:eq(5) td").html();
				String postalCode = doc.select("#section_content > table tr:eq(6) td").html();
				
				ipAddress.setDecimal(decimal);
				ipAddress.setHostname(hostname);
				ipAddress.setIsp(isp);
				ipAddress.setOrganization(organization);
				ipAddress.setServices(services);
				ipAddress.setType(type);
				ipAddress.setAssignment(assignment);
				ipAddress.setCountry(country);
				ipAddress.setRegion(region);
				ipAddress.setCity(city);
				ipAddress.setLatitude(latitude);
				ipAddress.setLongitude(longitude);
				ipAddress.setAreaCode(areaCode);
				ipAddress.setPostalCode(postalCode);
				
				//update data
				ipAddressModel.insert(ipAddress);
				
			}
			
			
		} catch (Exception e) {
			System.out.println("Can't get ip info for domain " + domain.getIpAddress());
			check.put("infoIp", "Can't get ip info for domain " + domain.getIpAddress());
		}
	}
	
	private void getGooglePageRank() {
        try {
            HTTPResponse response = (HTTPResponse) fGoogleRank.get();
            
            Document doc = Jsoup.parse(this.getContentCharset(response));
            int googlePageRank = 0;
            String google = doc.select(".show").select("span").first().select("font").text().replaceAll("[^0-9]", "");
            try {
                if(google!= null && !google.isEmpty())
                {
                    googlePageRank = Integer.parseInt(google);
                }
            } catch (Exception e) {
                check.put("googlePageRank", "Can't get position google page rank of domain " + domain.getDomain());
            }
            
            this.domain.setGooglePageRank(googlePageRank);
            
        } catch (Exception e) {
            check.put("googlePageRank", "Can't get position google page rank of domain " + domain.getDomain());
        }
    }
	
	private void getAgeDomain() {
        try {
            HTTPResponse response = (HTTPResponse) fAgeDomain.get();
            
            Document doc = Jsoup.parse(this.getContentCharset(response));
            String age = doc.select("div[class=KonaBody]").select("p").text();
               String[] ageSplit = age.split("(approximately)");
               if(ageSplit.length ==2)
               {
                   this.domain.setAge(ageSplit[1].replaceAll("[):]+", ""));
               }
            
        } catch (Exception e) {
            check.put("age", "Can't get age of domain " + domain.getDomain());
        }
    }
	
	private void getValidator() {
        try {
            HTTPResponse response = (HTTPResponse) fValidator.get();
            
            Document doc = Jsoup.parse(this.getContentCharset(response));
            
            String invalid = doc.select("#results_container .header .invalid").text();
            
            
            doc.select("#result ol#error_loop p.helpwanted").remove();
            String strValidError = doc.select("#result ol#error_loop").html();
            
            strValidError = strValidError.replaceAll("[<]{1}(a)[^>]+[>]{1}", "").replaceAll("<\\/a>", "");
            
            String[] str = invalid.split(",");
            
            if (str.length > 0) {
                this.domain.setValidError(Integer.parseInt(str[0].replaceAll("[\\D]*", "")));
            }
            
            if (str.length > 1) {
                this.domain.setValidWarnings(Integer.parseInt(str[1].replaceAll("[\\D]*", "")));
            }
            
            this.domain.setStrValidError(strValidError);
        } catch (Exception e) {
            check.put("validator", "Can't get validator of domain " + domain.getDomain());
        }
    }
	
	private void getDns() {
    	try {
    		HTTPResponse response = (HTTPResponse) fDns.get();
            Document doc = Jsoup.parse(this.getContentCharset(response));
            if (doc.select("#content .dns .soa-record") != null) {
            	String soaRecord = doc.select("#content .dns .soa-record").html();
            	this.domain.setSoaRecord(soaRecord);
            }
            
            if (doc.select("#content .dns .dns-record") != null) {
            	String dnsRecords = doc.select("#content .dns .dns-record").html();
            	this.domain.setDnsRecords(dnsRecords);
            }
            
		} catch (Exception e) {
			check.put("dns", "Can't get dns info for domain " + domain.getDomain());
		}
    }
	
	private void getOutlink(String content) {
        try {
            Document doc = Jsoup.parse(content);
            Elements link = doc.select("a");
            
            List<String> listLink = new ArrayList<String>();
            String check = null;
            for(int i=0;i<link.size();i++)
            {
                check = link.get(i).attr("rel"); 
                if(Pattern.matches("^http[\\w\\W]+", link.get(i).attr("href")))
                {
                    if( check != null && !check.equals("nofollow") || check == null)
                    {
                        Pattern p = Pattern.compile("^http[^\\/]+[\\/]{2}[^\\/]+");
    
                        Matcher m = p.matcher(link.get(i).attr("href"));
                        while (m.find()) {
                        	String name = UrlHelper.fomatDomains(m.group());
                            if(!listLink.contains(name))
                                listLink.add(name);
                        }
                    }
                }
            }
            
            this.domain.setOutlink(listLink);
            
        } catch (Exception e) {
            check.put("inlink", "Can't get outlink of domain " + domain.getDomain());
        }
    }
	
	/********************************************* LIBRARY FUNCTION *********************************************/
	private String getContentCharset(HTTPResponse response) {
        String charset = "UTF-8";
        String content = "";
        try {
			content = new String(response.getContent(),charset);
			Document doc = Jsoup.parse(content);
	        
	        int index = doc.select("meta").toString().toLowerCase().indexOf("charset=");
	        if (index != -1) {
	        	charset = doc.select("meta").toString().substring(index + 8).replaceAll("[^\\'\\\"0-9a-zA-Z\\-\\_]+[\\w\\W]+", "").replaceAll("[\\'\\\"]+", "");
	        }
	        content = new String(response.getContent(),charset);
	        
		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			//e.printStackTrace();
		}
        
    	return content;
    }
}
